import java.io.*;
import java.util.*;

public class WF {
    //字母出现频率
    public void FrequencyOfLetter(String path) throws IOException {
        BufferedReader br = new BufferedReader(new FileReader(path));
        String text = "";
        String article = "";
        while ((text=br.readLine())!= null){
            article += text;
        }
        String str = article.toLowerCase(); // 将stringBuffer转为字符并转换为小写
        float all = 0;
        char letter[] = { 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k',
                'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w',
                'x', 'y', 'z' };
        float[] count ={0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
        for (int i=0;i<letter.length;i++)
            for(int j=0;j<str.length();j++){
                if(str.charAt(j) == letter[i]){
                    ++count[i];
                }
            }
        for(int i=0;i<count.length;i++){
            all += count[i];
        }

        Map<Character,Float> map = new HashMap<Character, Float>();
        for(int i=0;i<count.length;i++){
            map.put(letter[i],count[i]);
        }
        List<Map.Entry<Character,Float>> list = new ArrayList<Map.Entry<Character,Float>>(map.entrySet());
        Comparator<Map.Entry<Character,Float>> comparator = new Comparator<Map.Entry<Character,Float>>() {
            public int compare(Map.Entry<Character,Float> left, Map.Entry<Character,Float> right) {
                return (left.getValue().compareTo(right.getValue()));
            }
        };
        // 集合默认升序升序
        Collections.sort(list,comparator);

        for(int i=0;i<list.size();i++){// 由高到低输出
            System.out.println(list.get(list.size()-i-1).getKey() +" "+String.format("%.0f",list.get(list.size()-i-1).getValue())+" "+String.format("%.2f",(list.get(list.size()-i-1).getValue())/all*100)+"%");
        }
    }

    //单词出现次数
    public void NumberOfWords(String path) throws IOException {
        BufferedReader br = new BufferedReader(new FileReader(path));

        StringBuffer sb = new StringBuffer();
        String text =null;
        while ((text=br.readLine())!= null){
            sb.append(text);// 将读取出的字符追加到stringbuffer中
        }
        br.close();  // 关闭读入流

        String str = sb.toString().toLowerCase(); // 将stringBuffer转为字符并转换为小写
        String[] words = str.split("[^(a-zA-Z)]+");  // 非单词的字符来分割，得到所有单词
        Map<String ,Integer> map = new HashMap<String, Integer>() ;

        for(String word :words){
            if(map.get(word)==null){  // 若不存在说明是第一次，则加入到map,出现次数为1
                map.put(word,1);
            }else{
                map.put(word,map.get(word)+1);  // 若存在，次数累加1
            }
        }

        // 排序
        List<Map.Entry<String ,Integer>> list = new ArrayList<Map.Entry<String,Integer>>(map.entrySet());

        Comparator<Map.Entry<String,Integer>> comparator = new Comparator<Map.Entry<String, Integer>>() {
            public int compare(Map.Entry<String, Integer> left, Map.Entry<String, Integer> right) {
                return (left.getValue().compareTo(right.getValue()));
            }
        };
        // 集合默认升序升序
        Collections.sort(list,comparator);

        for(int i=0;i<list.size();i++){// 由高到低输出
            System.out.println(list.get(list.size()-i-1).getKey() +":"+list.get(list.size()-i-1).getValue());
        }

    }

    //遍历文件夹中文件得到单词出现次数
    public void ENumberOfWords(String path) throws IOException {
        File file = new File(path);
        File []array =file.listFiles();
        for(int i=0;i<array.length;i++) {
            //判断是否是目录,只对文件执行操作
            if(array[i].isDirectory()) {
                continue;
            }else {
                System.out.println(array[i].getPath());
                NumberOfWords(array[i].getPath());
            }
        }
    }

    //统计前x个单词
    public void LNumberOfWords(String path,int x) throws IOException {
        BufferedReader br = new BufferedReader(new FileReader(path));

        StringBuffer sb = new StringBuffer();
        String text =null;
        while ((text=br.readLine())!= null){
            sb.append(text);// 将读取出的字符追加到stringbuffer中
        }
        br.close();  // 关闭读入流

        String str = sb.toString().toLowerCase(); // 将stringBuffer转为字符并转换为小写
        String[] words = str.split("[^(a-zA-Z)]+");  // 非单词的字符来分割，得到所有单词
        Map<String ,Integer> map = new HashMap<String, Integer>() ;

        for(String word :words){
            if(map.get(word)==null){  // 若不存在说明是第一次，则加入到map,出现次数为1
                map.put(word,1);
            }else{
                map.put(word,map.get(word)+1);  // 若存在，次数累加1
            }
        }

        // 排序
        List<Map.Entry<String ,Integer>> list = new ArrayList<Map.Entry<String,Integer>>(map.entrySet());

        Comparator<Map.Entry<String,Integer>> comparator = new Comparator<Map.Entry<String, Integer>>() {
            public int compare(Map.Entry<String, Integer> left, Map.Entry<String, Integer> right) {
                return (left.getValue().compareTo(right.getValue()));
            }
        };
        // 集合默认升序升序
        Collections.sort(list,comparator);

        for(int i=0;i<x;i++){// 由高到低输出
            System.out.println(list.get(list.size()-i-1).getKey() +":"+list.get(list.size()-i-1).getValue());
        }

    }

    //去除文本中的停用词
    public void RemoveOfStopWords(String stopWordFilePath,String filePath) throws IOException{
        File file1 = new File(stopWordFilePath);
        File file2 = new File(filePath);
        FileReader fr1 = new FileReader(file1);
        FileReader fr2 = new FileReader(file2);
        BufferedReader br1 = new BufferedReader(fr1);
        BufferedReader br2 = new BufferedReader(fr2);
        Map<String, Integer> map1 = new TreeMap<String, Integer>();
        Map<String, Integer> map2 = new TreeMap<String, Integer>();
        String line = "";
        String[] strs;
        while ((line=br1.readLine()) != null) {
            strs= line.toLowerCase().split("[^a-z0-9]+");
            for (int i = 0; i < strs.length; i++) {
                if(strs[i].equals("") || strs[i].toCharArray()[0]<='9'){
                    continue;
                }
                if (!map1.containsKey(strs[i])) {
                    map1.put(strs[i], 1);
                } else {
                    map1.put(strs[i], map1.get(strs[i]) + 1);
                }
            }
        }
        line = "";
        String[] strs1;
        while ((line = br2.readLine()) != null) {
            strs1= line.toLowerCase().split("[^a-z0-9]+");
            for (int i = 0; i < strs1.length; i++) {
                if(strs1[i].equals("") || strs1[i].toCharArray()[0]<='9'){
                    continue;
                }
                if (!map2.containsKey(strs1[i])) {
                    map2.put(strs1[i],1);
                } else {
                    continue;
                }
            }
        }
        //排序
        List<Map.Entry<String, Integer>> list = new ArrayList<Map.Entry<String, Integer>>(map2.entrySet());
        Collections.sort(list, new Comparator<Map.Entry<String, Integer>>() {
            public int compare(Map.Entry<String, Integer> o1, Map.Entry<String, Integer> o2) {
                if (o1.getValue() == o2.getValue()) {
                    return o1.getKey().compareTo(o2.getKey());
                }
                return o2.getValue().compareTo(o1.getValue());
            }
        });
        for (Map.Entry<String, Integer> entry : list) {
            if(map1.containsKey(entry.getKey())){}
            else{
                System.out.println(entry.getKey()+":"+entry.getValue());
            }
        }
    }

    //找出文章中可能存在的短语
    public void FindPhrase(Integer n) {

        try {
//            String fileName = "D:\\WorkSpace\\work\\软件工程\\one.txt"; // 文件名字
            FileReader fr = new FileReader("D:\\test\\one.txt");
            BufferedReader bufr = new BufferedReader(fr);
            Map<String, Integer> count = new TreeMap<String, Integer>();
            String s = "";
            String text = "";

            while ((s = bufr.readLine()) != null) {
                text = s.toLowerCase();

                //替换掉除空格外的所有标点符号，并且按照空格分隔文章
                text = text.replaceAll("[^0-9a-zA-Z\\s+]+", " ### ");
                String words[] = text.split("\\s+");//分割一个或多个空格

                //按规定数量找到短语
                for (int i = 0; i < words.length; i++) {
                    String sentence = "";
                    int j;
                    for(j = i; j < n + i && j < words.length; j++) {
                        if(words[j].equals("")) break;
                        sentence += words[j] + " ";
                    }
                    if(sentence.contains("###")||sentence.equals("")||j!=n+i) {}//包含###的字符串不符合要求，去除
                    else {
                        int cnt = 1;
                        if (count.get(sentence) != null) cnt = count.get(sentence) + 1;
                        count.put(sentence, cnt);
                    }
                }
            }



            //对treemap进行排序
            List<Map.Entry<String, Integer>> list = new ArrayList<Map.Entry<String, Integer>>(count.entrySet());
            Collections.sort(list, new Comparator<Map.Entry<String, Integer>>() {
                @Override
                public int compare(Map.Entry<String, Integer> o1, Map.Entry<String, Integer> o2) {
                    if (o1.getValue().compareTo(o2.getValue()) == 0) {
                        return o1.getKey().compareTo(o2.getKey());
                    }
                    return o2.getValue().compareTo(o1.getValue());
                }
            });

            int c=0;
            for(Map.Entry<String,Integer> entry : list){//输出短语及其数量,c也可控制输出数量
                if(c==10)
                    break;
                else{
                    c++;
                    System.out.println(entry.getKey()+":"+entry.getValue());
                }
            }

            fr.close();
            bufr.close();
        } catch (Exception e) {
            e.printStackTrace();
        }

    }

    //统一动词后统计单词出现次数
    public void UnifiedVerb(String v){
        File file1 = new File(v);
        File file2 = new File("D:\\test\\one.txt");

        try {
            FileReader fr1 = new FileReader(file1);
            FileReader fr2 = new FileReader(file2);

            BufferedReader bur1 = new BufferedReader(fr1);
            BufferedReader bur2 = new BufferedReader(fr2);

            Map<String, Integer> imap = new TreeMap<String, Integer>();
            Map<String, Integer> nmap = new TreeMap<String, Integer>();

            //对动词表按行存入
            String line = "";
            while ((line=bur1.readLine()) != null) {
                line= "#"+line.toLowerCase().replaceAll("[^a-z0-9]+","#")+"#";
                nmap.put(line,1);
            }


            List<Map.Entry<String, Integer>> list1 = new ArrayList<Map.Entry<String, Integer>>(nmap.entrySet());
            line = "";
            String[] strs1;
            while ((line = bur2.readLine()) != null) {
                strs1= line.toLowerCase().split("[^a-z0-9]+");
                for (int i = 0; i < strs1.length; i++) {
                    if(strs1[i].equals("") || strs1[i].toCharArray()[0]<='9'){
                        continue;
                    }

                    String aaa ="";
                    for(Map.Entry<String,Integer> nentry:list1){
                        //判断是否包含了此单词，包含则进行分割并存入第一个单词，即动词原形
                        String a = nentry.getKey().toString();
                        if(a.contains("#"+strs1[i]+"#")){
                            String[] aa = a.split("[^a-z0-9]+");
                            aaa = aa[1];
                            break;
                        }

                    }

                    //存入
                    if(!aaa.equals("")){
                        if (!imap.containsKey(aaa)) {
                            imap.put(aaa,1);
                        } else {
                            imap.put(aaa,imap.get(aaa)+1);
                        }
                    }else{
                        if (!imap.containsKey(strs1[i])) {
                            imap.put(strs1[i],1);
                        } else {
                            imap.put(strs1[i],imap.get(strs1[i])+1);
                        }
                    }

                }
            }

            //排序
            List<Map.Entry<String, Integer>> list = new ArrayList<Map.Entry<String, Integer>>(imap.entrySet());
            Collections.sort(list, new Comparator<Map.Entry<String, Integer>>() {
                public int compare(Map.Entry<String, Integer> o1, Map.Entry<String, Integer> o2) {
                    if (o1.getValue() == o2.getValue()) {
                        return o1.getKey().compareTo(o2.getKey());
                    }
                    return o2.getValue().compareTo(o1.getValue());
                }
            });

            //输出
            for (Map.Entry<String, Integer> entry : list) {
                System.out.println(entry.getKey()+":"+entry.getValue());
            }

            fr1.close();
            fr2.close();
            bur1.close();
            bur2.close();

        } catch (Exception e) {
            e.printStackTrace();
        }

    }

    public static void main(String [] args) throws Exception {
        WF wf = new WF();
//        wf.FrequencyOfLetter("D:\\test\\one.txt");
//        wf.NumberOfWords("D:\\test\\one.txt");
//        wf.ENumberOfWords("D:\\test\\600篇英文小说");
//          wf.LNumberOfWords("D:\\test\\600篇英文小说",10);
//        wf.RemoveOfStopWords("D:\\test\\stopwords_en.txt","D:\\test\\one.txt");
//            wf.FindPhrase(2);
        wf.UnifiedVerb("D:\\test\\irregular verbs.txt");


    }
}
